
/**
 * text split into chunks
 * chunkLen - one chunk len. max: 3500
 * overlapLen - The size of the before and after Text
 * chunkLen > overlapLen
 * markdown
 */
var splitText2Chunks = function (props) {
    var _a = props.text, text = _a === void 0 ? '' : _a, chunkLen = props.chunkLen, _b = props.overlapRatio, overlapRatio = _b === void 0 ? 0.2 : _b;
    var splitMarker = 'SPLIT_HERE_SPLIT_HERE';
    var overlapLen = Math.round(chunkLen * overlapRatio);
    // The larger maxLen is, the next sentence is less likely to trigger splitting
    var stepReges = [
        { reg: /^(#\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
        { reg: /^(##\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
        { reg: /^(###\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
        { reg: /^(####\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
        { reg: /([\n]{2})/g, maxLen: chunkLen * 1.4 },
        { reg: /([\n](?![\*\-|>`0-9]))/g, maxLen: chunkLen * 1.8 }, // (?![\*\-|>`0-9]): markdown special char
        { reg: /([\n])/g, maxLen: chunkLen * 1.4 },
        { reg: /([。]|([a-zA-Z])\.\s)/g, maxLen: chunkLen * 1.4 },
        { reg: /([！]|!\s)/g, maxLen: chunkLen * 1.4 },
        { reg: /([？]|\?\s)/g, maxLen: chunkLen * 1.6 },
        { reg: /([；]|;\s)/g, maxLen: chunkLen * 1.8 },
        { reg: /([，]|,\s)/g, maxLen: chunkLen * 2 }
    ];
    var getSplitTexts = function (_a) {
        var text = _a.text, step = _a.step;
        if (step >= stepReges.length) {
            return [text];
        }
        var isMarkdownSplit = step <= 3;
        var reg = stepReges[step].reg;
        var splitTexts = text
            .replace(reg, isMarkdownSplit ? "".concat(splitMarker, "$1") : "$1".concat(splitMarker))
            .split("".concat(splitMarker))
            .filter(function (part) { return part.trim(); });
        
        return splitTexts;
    };
    var getOneTextOverlapText = function (_a) {
        var text = _a.text, step = _a.step;
        var forbidOverlap = step <= 6;
        var maxOverlapLen = chunkLen * 0.4;
        // step >= stepReges.length: Do not overlap incomplete sentences
        if (forbidOverlap || overlapLen === 0 || step >= stepReges.length)
            return '';
        var splitTexts = getSplitTexts({ text: text, step: step });
        var overlayText = '';
        for (var i = splitTexts.length - 1; i >= 0; i--) {
            var currentText = splitTexts[i];
            var newText = currentText + overlayText;
            var newTextLen = newText.length;
            if (newTextLen > overlapLen) {
                if (newTextLen > maxOverlapLen) {
                    var text_1 = getOneTextOverlapText({ text: newText, step: step + 1 });
                    return text_1 || overlayText;
                }
                return newText;
            }
            overlayText = newText;
        }
        return overlayText;
    };
    var splitTextRecursively = function (_a) {
        var _b = _a.text, text = _b === void 0 ? '' : _b, step = _a.step, lastText = _a.lastText;
        // mini text
        if (text.length <= chunkLen) {
            return [text];
        }

        
        // oversize
        if (step >= stepReges.length) {
            
            if (text.length < chunkLen * 3) {
                return [text];
            }
            // use slice-chunkLen to split text
            var chunks_1 = [];
            for (var i = 0; i < text.length; i += chunkLen - overlapLen) {
                
                chunks_1.push(text.slice(i, i + chunkLen));
            }
            return chunks_1;
        }
        var maxLen = stepReges[step].maxLen;
        var minChunkLen = chunkLen * 0.7;
        // split text by special char
        var splitTexts = getSplitTexts({ text: text, step: step });
        var chunks = [];
        for (var i = 0; i < splitTexts.length; i++) {
            var currentText = splitTexts[i];
            var currentTextLen = currentText.length;
            var lastTextLen = lastText.length;
            var newText = lastText + currentText;
            var newTextLen = lastTextLen + currentTextLen;
            // newText is too large(now, The lastText must be smaller than chunkLen)
            if (newTextLen > maxLen) {
                // lastText greater minChunkLen, direct push it to chunks, not add to next chunk. (large lastText)
                if (lastTextLen > minChunkLen) {
                    chunks.push(lastText);
                    lastText = getOneTextOverlapText({ text: lastText, step: step }); // next chunk will start with overlayText
                    i--;
                    continue;
                }
                // split new Text, split chunks must will greater 1 (small lastText)
                var innerChunks = splitTextRecursively({
                    text: newText,
                    step: step + 1,
                    lastText: ''
                });
                var lastChunk = innerChunks[innerChunks.length - 1];
                // last chunk is too small, concat it to lastText
                if (lastChunk.length < minChunkLen) {
                    chunks.push.apply(chunks, innerChunks.slice(0, -1));
                    lastText = lastChunk;
                }
                else {
                    chunks.push.apply(chunks, innerChunks);
                    // compute new overlapText
                    lastText = getOneTextOverlapText({
                        text: lastChunk,
                        step: step
                    });
                }
                continue;
            }
            // size less than chunkLen, push text to last chunk. now, text definitely less than maxLen
            lastText = newText;
            // If the chunk size reaches, add a chunk
            if (newTextLen >= chunkLen) {
                chunks.push(lastText);
                lastText = getOneTextOverlapText({ text: lastText, step: step });
            }
        }
        /* If the last chunk is independent, it needs to be push chunks. */
        if (lastText && chunks[chunks.length - 1] && !chunks[chunks.length - 1].endsWith(lastText)) {
            if (lastText.length < chunkLen * 0.4) {
                chunks[chunks.length - 1] = chunks[chunks.length - 1] + lastText;
            }
            else {
                chunks.push(lastText);
            }
        }
        return chunks;
    };
    try {
        var chunks = splitTextRecursively({
            text: text,
            step: 0,
            lastText: ''
        });
        return {
            chunks: chunks
        };
    }
    catch (err) {
        throw new Error();
    }
};

